{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot import Collection\n",
    "from cot.stats import evaluation_as_table"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 167     | -      |\n",
       "| med_qa         | -       | -       | 167    |\n",
       "| medmc_qa       | -       | 167     | -      |\n",
       "| open_book_qa   | -       | -       | 167    |\n",
       "| strategy_qa    | 167     | -       | -      |\n",
       "| worldtree      | -       | -       | 167    |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll = Collection.from_json(\"ts_1k_empty.json\")\n",
    "coll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Configuration of the input and parameters of the language model \n",
    "config={\n",
    "    \"instruction_keys\": None,\n",
    "    \"cot_trigger_keys\": [\"kojima-01\"],\n",
    "    \"answer_extraction_keys\": 'auto-kojima', \n",
    "    \"author\" : \"thoughtsource\",\n",
    "    \"api_service\": \"cohere\",\n",
    "    \"api_time_interval\": 1,\n",
    "    \"engine\": \"command-xlarge-nightly\", \n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 512,\n",
    "    \"verbose\": False,\n",
    "    \"warn\": False,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26ff1273cd634c1ca50f7174e4df057b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/167 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(API-)Error in item 0: HTTPSConnectionPool(host='api.cohere.ai', port=443): Max retries exceeded with url: /generate (Caused by ResponseError('too many 429 error responses'))\n",
      "Retrying with additional time of 10 seconds.\n",
      "(API-)Error in item 0: HTTPSConnectionPool(host='api.cohere.ai', port=443): Max retries exceeded with url: /generate (Caused by ResponseError('too many 429 error responses'))\n",
      "Retrying with additional time of 20 seconds.\n",
      "(API-)Error in item 0: HTTPSConnectionPool(host='api.cohere.ai', port=443): Max retries exceeded with url: /generate (Caused by ResponseError('too many 429 error responses'))\n",
      "Retrying with additional time of 30 seconds.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[17], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m coll\u001b[39m.\u001b[39;49mgenerate(config\u001b[39m=\u001b[39;49mconfig)\n",
      "File \u001b[0;32m~/work/ThoughtSource/libs/cot/cot/dataloader.py:409\u001b[0m, in \u001b[0;36mCollection.generate\u001b[0;34m(self, name, split, config)\u001b[0m\n\u001b[1;32m    407\u001b[0m         \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mGenerating \u001b[39m\u001b[39m{\u001b[39;00mname\u001b[39m}\u001b[39;00m\u001b[39m...\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    408\u001b[0m         \u001b[39mfor\u001b[39;00m split \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_cache[name]:\n\u001b[0;32m--> 409\u001b[0m             \u001b[39mself\u001b[39m[name][split] \u001b[39m=\u001b[39m generate_and_extract(\u001b[39mself\u001b[39;49m[name][split], config\u001b[39m=\u001b[39;49mconfig)\n\u001b[1;32m    410\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    411\u001b[0m     \u001b[39mif\u001b[39;00m split \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/work/ThoughtSource/libs/cot/cot/generate.py:63\u001b[0m, in \u001b[0;36mgenerate_and_extract\u001b[0;34m(data, config)\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[39m# The config is transformed into a dataclass object, where all testing is done\u001b[39;00m\n\u001b[1;32m     60\u001b[0m \u001b[39m# But it will be transformed back to a dictionary for the function 'map'\u001b[39;00m\n\u001b[1;32m     61\u001b[0m config_as_dataclass \u001b[39m=\u001b[39m Config(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39madaptive_config)\n\u001b[0;32m---> 63\u001b[0m \u001b[39mreturn\u001b[39;00m data\u001b[39m.\u001b[39;49mmap(\n\u001b[1;32m     64\u001b[0m     _generate_and_extract,\n\u001b[1;32m     65\u001b[0m     with_indices\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[1;32m     66\u001b[0m     fn_kwargs\u001b[39m=\u001b[39;49masdict(config_as_dataclass),\n\u001b[1;32m     67\u001b[0m     features\u001b[39m=\u001b[39;49mfeatures,\n\u001b[1;32m     68\u001b[0m     load_from_cache_file\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m     69\u001b[0m )\n",
      "File \u001b[0;32m~/work/ThoughtSource/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:1955\u001b[0m, in \u001b[0;36mDataset.map\u001b[0;34m(self, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint, desc)\u001b[0m\n\u001b[1;32m   1952\u001b[0m disable_tqdm \u001b[39m=\u001b[39m \u001b[39mnot\u001b[39;00m logging\u001b[39m.\u001b[39mis_progress_bar_enabled()\n\u001b[1;32m   1954\u001b[0m \u001b[39mif\u001b[39;00m num_proc \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mor\u001b[39;00m num_proc \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[0;32m-> 1955\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_map_single(\n\u001b[1;32m   1956\u001b[0m         function\u001b[39m=\u001b[39;49mfunction,\n\u001b[1;32m   1957\u001b[0m         with_indices\u001b[39m=\u001b[39;49mwith_indices,\n\u001b[1;32m   1958\u001b[0m         with_rank\u001b[39m=\u001b[39;49mwith_rank,\n\u001b[1;32m   1959\u001b[0m         input_columns\u001b[39m=\u001b[39;49minput_columns,\n\u001b[1;32m   1960\u001b[0m         batched\u001b[39m=\u001b[39;49mbatched,\n\u001b[1;32m   1961\u001b[0m         batch_size\u001b[39m=\u001b[39;49mbatch_size,\n\u001b[1;32m   1962\u001b[0m         drop_last_batch\u001b[39m=\u001b[39;49mdrop_last_batch,\n\u001b[1;32m   1963\u001b[0m         remove_columns\u001b[39m=\u001b[39;49mremove_columns,\n\u001b[1;32m   1964\u001b[0m         keep_in_memory\u001b[39m=\u001b[39;49mkeep_in_memory,\n\u001b[1;32m   1965\u001b[0m         load_from_cache_file\u001b[39m=\u001b[39;49mload_from_cache_file,\n\u001b[1;32m   1966\u001b[0m         cache_file_name\u001b[39m=\u001b[39;49mcache_file_name,\n\u001b[1;32m   1967\u001b[0m         writer_batch_size\u001b[39m=\u001b[39;49mwriter_batch_size,\n\u001b[1;32m   1968\u001b[0m         features\u001b[39m=\u001b[39;49mfeatures,\n\u001b[1;32m   1969\u001b[0m         disable_nullable\u001b[39m=\u001b[39;49mdisable_nullable,\n\u001b[1;32m   1970\u001b[0m         fn_kwargs\u001b[39m=\u001b[39;49mfn_kwargs,\n\u001b[1;32m   1971\u001b[0m         new_fingerprint\u001b[39m=\u001b[39;49mnew_fingerprint,\n\u001b[1;32m   1972\u001b[0m         disable_tqdm\u001b[39m=\u001b[39;49mdisable_tqdm,\n\u001b[1;32m   1973\u001b[0m         desc\u001b[39m=\u001b[39;49mdesc,\n\u001b[1;32m   1974\u001b[0m     )\n\u001b[1;32m   1975\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m   1977\u001b[0m     \u001b[39mdef\u001b[39;00m \u001b[39mformat_cache_file_name\u001b[39m(cache_file_name, rank):\n",
      "File \u001b[0;32m~/work/ThoughtSource/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:520\u001b[0m, in \u001b[0;36mtransmit_tasks.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    518\u001b[0m     \u001b[39mself\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m=\u001b[39m kwargs\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mself\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    519\u001b[0m \u001b[39m# apply actual function\u001b[39;00m\n\u001b[0;32m--> 520\u001b[0m out: Union[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mDatasetDict\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    521\u001b[0m datasets: List[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(out\u001b[39m.\u001b[39mvalues()) \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(out, \u001b[39mdict\u001b[39m) \u001b[39melse\u001b[39;00m [out]\n\u001b[1;32m    522\u001b[0m \u001b[39mfor\u001b[39;00m dataset \u001b[39min\u001b[39;00m datasets:\n\u001b[1;32m    523\u001b[0m     \u001b[39m# Remove task templates if a column mapping of the template is no longer valid\u001b[39;00m\n",
      "File \u001b[0;32m~/work/ThoughtSource/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:487\u001b[0m, in \u001b[0;36mtransmit_format.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    480\u001b[0m self_format \u001b[39m=\u001b[39m {\n\u001b[1;32m    481\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mtype\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_format_type,\n\u001b[1;32m    482\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mformat_kwargs\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_format_kwargs,\n\u001b[1;32m    483\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mcolumns\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_format_columns,\n\u001b[1;32m    484\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39moutput_all_columns\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_output_all_columns,\n\u001b[1;32m    485\u001b[0m }\n\u001b[1;32m    486\u001b[0m \u001b[39m# apply actual function\u001b[39;00m\n\u001b[0;32m--> 487\u001b[0m out: Union[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mDatasetDict\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    488\u001b[0m datasets: List[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(out\u001b[39m.\u001b[39mvalues()) \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(out, \u001b[39mdict\u001b[39m) \u001b[39melse\u001b[39;00m [out]\n\u001b[1;32m    489\u001b[0m \u001b[39m# re-apply format to the output\u001b[39;00m\n",
      "File \u001b[0;32m~/work/ThoughtSource/venv/lib/python3.10/site-packages/datasets/fingerprint.py:458\u001b[0m, in \u001b[0;36mfingerprint_transform.<locals>._fingerprint.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    452\u001b[0m             kwargs[fingerprint_name] \u001b[39m=\u001b[39m update_fingerprint(\n\u001b[1;32m    453\u001b[0m                 \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_fingerprint, transform, kwargs_for_fingerprint\n\u001b[1;32m    454\u001b[0m             )\n\u001b[1;32m    456\u001b[0m \u001b[39m# Call actual function\u001b[39;00m\n\u001b[0;32m--> 458\u001b[0m out \u001b[39m=\u001b[39m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    460\u001b[0m \u001b[39m# Update fingerprint of in-place transforms + update in-place history of transforms\u001b[39;00m\n\u001b[1;32m    462\u001b[0m \u001b[39mif\u001b[39;00m inplace:  \u001b[39m# update after calling func so that the fingerprint doesn't change if the function fails\u001b[39;00m\n",
      "File \u001b[0;32m~/work/ThoughtSource/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:2320\u001b[0m, in \u001b[0;36mDataset._map_single\u001b[0;34m(self, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, new_fingerprint, rank, offset, disable_tqdm, desc, cache_only)\u001b[0m\n\u001b[1;32m   2318\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m batched:\n\u001b[1;32m   2319\u001b[0m     \u001b[39mfor\u001b[39;00m i, example \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(pbar):\n\u001b[0;32m-> 2320\u001b[0m         example \u001b[39m=\u001b[39m apply_function_on_filtered_inputs(example, i, offset\u001b[39m=\u001b[39;49moffset)\n\u001b[1;32m   2321\u001b[0m         \u001b[39mif\u001b[39;00m update_data:\n\u001b[1;32m   2322\u001b[0m             \u001b[39mif\u001b[39;00m i \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m:\n",
      "File \u001b[0;32m~/work/ThoughtSource/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:2220\u001b[0m, in \u001b[0;36mDataset._map_single.<locals>.apply_function_on_filtered_inputs\u001b[0;34m(inputs, indices, check_same_num_examples, offset)\u001b[0m\n\u001b[1;32m   2218\u001b[0m \u001b[39mif\u001b[39;00m with_rank:\n\u001b[1;32m   2219\u001b[0m     additional_args \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m (rank,)\n\u001b[0;32m-> 2220\u001b[0m processed_inputs \u001b[39m=\u001b[39m function(\u001b[39m*\u001b[39;49mfn_args, \u001b[39m*\u001b[39;49madditional_args, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mfn_kwargs)\n\u001b[1;32m   2221\u001b[0m \u001b[39mif\u001b[39;00m update_data \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m   2222\u001b[0m     \u001b[39m# Check if the function returns updated examples\u001b[39;00m\n\u001b[1;32m   2223\u001b[0m     update_data \u001b[39m=\u001b[39m \u001b[39misinstance\u001b[39m(processed_inputs, (Mapping, pa\u001b[39m.\u001b[39mTable))\n",
      "File \u001b[0;32m~/work/ThoughtSource/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:1915\u001b[0m, in \u001b[0;36mDataset.map.<locals>.decorate.<locals>.decorated\u001b[0;34m(item, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1911\u001b[0m decorated_item \u001b[39m=\u001b[39m (\n\u001b[1;32m   1912\u001b[0m     Example(item, features\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfeatures) \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m batched \u001b[39melse\u001b[39;00m Batch(item, features\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfeatures)\n\u001b[1;32m   1913\u001b[0m )\n\u001b[1;32m   1914\u001b[0m \u001b[39m# Use the LazyDict internally, while mapping the function\u001b[39;00m\n\u001b[0;32m-> 1915\u001b[0m result \u001b[39m=\u001b[39m f(decorated_item, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1916\u001b[0m \u001b[39m# Return a standard dict\u001b[39;00m\n\u001b[1;32m   1917\u001b[0m \u001b[39mreturn\u001b[39;00m result\u001b[39m.\u001b[39mdata \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(result, LazyDict) \u001b[39melse\u001b[39;00m result\n",
      "File \u001b[0;32m~/work/ThoughtSource/libs/cot/cot/generate.py:162\u001b[0m, in \u001b[0;36m_generate_and_extract\u001b[0;34m(item, idx, idx_range, author, api_service, engine, temperature, max_tokens, api_time_interval, instruction_keys, cot_trigger_keys, template_cot_generation, answer_extraction_keys, template_answer_extraction, warn, verbose)\u001b[0m\n\u001b[1;32m    159\u001b[0m     \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m-----------------COT TRIGGER TEXT-----------------\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    160\u001b[0m     \u001b[39mprint\u001b[39m(generate_cot_prompt)\n\u001b[0;32m--> 162\u001b[0m cot \u001b[39m=\u001b[39m query_model(\n\u001b[1;32m    163\u001b[0m     generate_cot_prompt,\n\u001b[1;32m    164\u001b[0m     api_service,\n\u001b[1;32m    165\u001b[0m     engine,\n\u001b[1;32m    166\u001b[0m     temperature,\n\u001b[1;32m    167\u001b[0m     max_tokens,\n\u001b[1;32m    168\u001b[0m     api_time_interval,\n\u001b[1;32m    169\u001b[0m )\n\u001b[1;32m    170\u001b[0m \u001b[39mif\u001b[39;00m verbose:\n\u001b[1;32m    171\u001b[0m     \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m------------------GENERATED COT-------------------\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "File \u001b[0;32m~/work/ThoughtSource/libs/cot/cot/generate.py:464\u001b[0m, in \u001b[0;36mquery_model\u001b[0;34m(input, api_service, engine, temperature, max_tokens, api_time_interval)\u001b[0m\n\u001b[1;32m    458\u001b[0m     \u001b[39m# return (\"This is a \" + 20 * \"long \" + \"Mock CoT.\\n\")*20\u001b[39;00m\n\u001b[1;32m    459\u001b[0m \n\u001b[1;32m    460\u001b[0m \u001b[39m# langchain package implementation\u001b[39;00m\n\u001b[1;32m    461\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    462\u001b[0m     \u001b[39mfrom\u001b[39;00m \u001b[39mlangchain\u001b[39;00m \u001b[39mimport\u001b[39;00m LLMChain, Prompt\n\u001b[0;32m--> 464\u001b[0m     time\u001b[39m.\u001b[39;49msleep(api_time_interval)\n\u001b[1;32m    465\u001b[0m     template \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m{prompt}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m    466\u001b[0m     prompt \u001b[39m=\u001b[39m Prompt(template\u001b[39m=\u001b[39mtemplate, input_variables\u001b[39m=\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39mprompt\u001b[39m\u001b[39m\"\u001b[39m])\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "coll.generate(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6cfc88404a80455c893b9ae0d127ccf3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40ecc225a82e435fb64d6e0476da3d85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f089cdf391be4b9d98940b6f0067e500",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "043e58dd2c384b988746fcc0637c0b14",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2f3239fe31c4c999395c696e285dd35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc7e509ef9e04607857b6d4b73d5207a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'command-xlarge-nightly': {'qa-08_None_kojima-A-E': 1.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'command-xlarge-nightly': {'qa-08_None_kojima-A-E': 0.0}}}},\n",
       " 'medmc_qa': {'validation': {'accuracy': {'command-xlarge-nightly': {'qa-08_None_kojima-A-D': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'command-xlarge-nightly': {'qa-08_None_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'command-xlarge-nightly': {'qa-08_None_kojima-yes-no': 0.0}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'command-xlarge-nightly': {'qa-08_None_kojima-A-D': 1.0}}}}}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join strings from a list with underscore\n",
    "def join_strings(list_of_strings):\n",
    "    if list_of_strings is None:\n",
    "        list_of_strings = [\"None\"]\n",
    "    if isinstance(list_of_strings, str):\n",
    "        list_of_strings = [list_of_strings]\n",
    "    joined_string = \"\"\n",
    "    for string in list_of_strings:\n",
    "        joined_string += (\"-\" + str(string))\n",
    "    # delete first underscore\n",
    "    joined_string = joined_string[1:]\n",
    "    return(joined_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll.dump(\"thoughtsource_167\" + \"_\" + config['api_service'] + \"_\" + config['engine'].replace(\"/\", \"_\") + \"_\" + join_strings(config[\"instruction_keys\"]) + \"_\" + join_strings(config[\"cot_trigger_keys\"]) + \".json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a19cb823e24c98ee05a9cfa4a3a579b5d56d4c1a735f2a12456750b95a1e155e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
